{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Load packages\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "from validation import compute_f1\n",
    "from keras.models import Model, load_model\n",
    "from keras.layers import TimeDistributed, Conv1D, Dense, Embedding, Input, Dropout, LSTM, Bidirectional, MaxPooling1D, \\\n",
    "    Flatten, concatenate\n",
    "from prepro import readfile, createBatches, createMatrices, iterate_minibatches, addCharInformation, padding\n",
    "from keras.utils import plot_model\n",
    "from keras.initializers import RandomUniform\n",
    "from keras.optimizers import SGD, Nadam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class initialised.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Initialise class\"\"\"\n",
    "\n",
    "class CNN_BLSTM(object):\n",
    "    \n",
    "    def __init__(self, EPOCHS, DROPOUT, DROPOUT_RECURRENT, LSTM_STATE_SIZE, CONV_SIZE, LEARNING_RATE, OPTIMIZER):\n",
    "        \n",
    "        self.epochs = EPOCHS\n",
    "        self.dropout = DROPOUT\n",
    "        self.dropout_recurrent = DROPOUT_RECURRENT\n",
    "        self.lstm_state_size = LSTM_STATE_SIZE\n",
    "        self.conv_size = CONV_SIZE\n",
    "        self.learning_rate = LEARNING_RATE\n",
    "        self.optimizer = OPTIMIZER\n",
    "        \n",
    "    def loadData(self):\n",
    "        \"\"\"Load data and add character information\"\"\"\n",
    "        self.trainSentences = readfile(\"data/train.txt\")\n",
    "        self.devSentences = readfile(\"data/dev.txt\")\n",
    "        self.testSentences = readfile(\"data/test.txt\")\n",
    "\n",
    "    def addCharInfo(self):\n",
    "        # format: [['EU', ['E', 'U'], 'B-ORG\\n'], ...]\n",
    "        self.trainSentences = addCharInformation(self.trainSentences)\n",
    "        self.devSentences = addCharInformation(self.devSentences)\n",
    "        self.testSentences = addCharInformation(self.testSentences)\n",
    "\n",
    "    def embed(self):\n",
    "        \"\"\"Create word- and character-level embeddings\"\"\"\n",
    "\n",
    "        labelSet = set()\n",
    "        words = {}\n",
    "\n",
    "        # unique words and labels in data  \n",
    "        for dataset in [self.trainSentences, self.devSentences, self.testSentences]:\n",
    "            for sentence in dataset:\n",
    "                for token, char, label in sentence:\n",
    "                    # token ... token, char ... list of chars, label ... BIO labels   \n",
    "                    labelSet.add(label)\n",
    "                    words[token.lower()] = True\n",
    "\n",
    "        # mapping for labels\n",
    "        self.label2Idx = {}\n",
    "        for label in labelSet:\n",
    "            self.label2Idx[label] = len(self.label2Idx)\n",
    "\n",
    "        # mapping for token cases\n",
    "        case2Idx = {'numeric': 0, 'allLower': 1, 'allUpper': 2, 'initialUpper': 3, 'other': 4, 'mainly_numeric': 5,\n",
    "                    'contains_digit': 6, 'PADDING_TOKEN': 7}\n",
    "        self.caseEmbeddings = np.identity(len(case2Idx), dtype='float32')  # identity matrix used \n",
    "\n",
    "        # read GLoVE word embeddings\n",
    "        word2Idx = {}\n",
    "        self.wordEmbeddings = []\n",
    "\n",
    "        fEmbeddings = open(\"embeddings/glove.6B.50d.txt\", encoding=\"utf-8\")\n",
    "\n",
    "        # loop through each word in embeddings\n",
    "        for line in fEmbeddings:\n",
    "            split = line.strip().split(\" \")\n",
    "            word = split[0]  # embedding word entry\n",
    "\n",
    "            if len(word2Idx) == 0:  # add padding+unknown\n",
    "                word2Idx[\"PADDING_TOKEN\"] = len(word2Idx)\n",
    "                vector = np.zeros(len(split) - 1)  # zero vector for 'PADDING' word\n",
    "                self.wordEmbeddings.append(vector)\n",
    "\n",
    "                word2Idx[\"UNKNOWN_TOKEN\"] = len(word2Idx)\n",
    "                vector = np.random.uniform(-0.25, 0.25, len(split) - 1)\n",
    "                self.wordEmbeddings.append(vector)\n",
    "\n",
    "            if split[0].lower() in words:\n",
    "                vector = np.array([float(num) for num in split[1:]])\n",
    "                self.wordEmbeddings.append(vector)  # word embedding vector\n",
    "                word2Idx[split[0]] = len(word2Idx)  # corresponding word dict\n",
    "\n",
    "        self.wordEmbeddings = np.array(self.wordEmbeddings)\n",
    "\n",
    "        # dictionary of all possible characters\n",
    "        self.char2Idx = {\"PADDING\": 0, \"UNKNOWN\": 1}\n",
    "        for c in \" 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.,-_()[]{}!?:;#'\\\"/\\\\%$`&=*+@^~|<>\":\n",
    "            self.char2Idx[c] = len(self.char2Idx)\n",
    "\n",
    "        # format: [[wordindices], [caseindices], [padded word indices], [label indices]]\n",
    "        self.train_set = padding(createMatrices(self.trainSentences, word2Idx, self.label2Idx, case2Idx, self.char2Idx))\n",
    "        self.dev_set = padding(createMatrices(self.devSentences, word2Idx, self.label2Idx, case2Idx, self.char2Idx))\n",
    "        self.test_set = padding(createMatrices(self.testSentences, word2Idx, self.label2Idx, case2Idx, self.char2Idx))\n",
    "\n",
    "        self.idx2Label = {v: k for k, v in self.label2Idx.items()}\n",
    "        \n",
    "    def createBatches(self):\n",
    "        \"\"\"Create batches\"\"\"\n",
    "        self.train_batch, self.train_batch_len = createBatches(self.train_set)\n",
    "        self.dev_batch, self.dev_batch_len = createBatches(self.dev_set)\n",
    "        self.test_batch, self.test_batch_len = createBatches(self.test_set)\n",
    "        \n",
    "    def tag_dataset(self, dataset, model):\n",
    "        \"\"\"Tag data with numerical values\"\"\"\n",
    "        correctLabels = []\n",
    "        predLabels = []\n",
    "        for i, data in enumerate(dataset):\n",
    "            tokens, casing, char, labels = data\n",
    "            tokens = np.asarray([tokens])\n",
    "            casing = np.asarray([casing])\n",
    "            char = np.asarray([char])\n",
    "            pred = model.predict([tokens, casing, char], verbose=False)[0]\n",
    "            pred = pred.argmax(axis=-1)  # Predict the classes\n",
    "            correctLabels.append(labels)\n",
    "            predLabels.append(pred)\n",
    "        return predLabels, correctLabels\n",
    "    \n",
    "    def buildModel(self):\n",
    "        \"\"\"Model layers\"\"\"\n",
    "\n",
    "        # character input\n",
    "        character_input = Input(shape=(None, 52,), name=\"Character_input\")\n",
    "        embed_char_out = TimeDistributed(\n",
    "            Embedding(len(self.char2Idx), 30, embeddings_initializer=RandomUniform(minval=-0.5, maxval=0.5)), name=\"Character_embedding\")(\n",
    "            character_input)\n",
    "\n",
    "        dropout = Dropout(self.dropout)(embed_char_out)\n",
    "\n",
    "        # CNN\n",
    "        conv1d_out = TimeDistributed(Conv1D(kernel_size=self.conv_size, filters=30, padding='same', activation='tanh', strides=1), name=\"Convolution\")(dropout)\n",
    "        maxpool_out = TimeDistributed(MaxPooling1D(52), name=\"Maxpool\")(conv1d_out)\n",
    "        char = TimeDistributed(Flatten(), name=\"Flatten\")(maxpool_out)\n",
    "        char = Dropout(self.dropout)(char)\n",
    "\n",
    "        # word-level input\n",
    "        words_input = Input(shape=(None,), dtype='int32', name='words_input')\n",
    "        words = Embedding(input_dim=self.wordEmbeddings.shape[0], output_dim=self.wordEmbeddings.shape[1], weights=[self.wordEmbeddings],\n",
    "                          trainable=False)(words_input)\n",
    "\n",
    "        # case-info input\n",
    "        casing_input = Input(shape=(None,), dtype='int32', name='casing_input')\n",
    "        casing = Embedding(output_dim=self.caseEmbeddings.shape[1], input_dim=self.caseEmbeddings.shape[0], weights=[self.caseEmbeddings],\n",
    "                           trainable=False)(casing_input)\n",
    "\n",
    "        # concat & BLSTM\n",
    "        output = concatenate([words, casing, char])\n",
    "        output = Bidirectional(LSTM(self.lstm_state_size, \n",
    "                                    return_sequences=True, \n",
    "                                    dropout=self.dropout,                        # on input to each LSTM block\n",
    "                                    recurrent_dropout=self.dropout_recurrent     # on recurrent input signal\n",
    "                                   ), name=\"BLSTM\")(output)\n",
    "        output = TimeDistributed(Dense(len(self.label2Idx), activation='softmax'),name=\"Softmax_layer\")(output)\n",
    "\n",
    "        # set up model\n",
    "        self.model = Model(inputs=[words_input, casing_input, character_input], outputs=[output])\n",
    "        \n",
    "        self.model.compile(loss='sparse_categorical_crossentropy', optimizer=self.optimizer)\n",
    "        \n",
    "        self.init_weights = self.model.get_weights()\n",
    "        \n",
    "        plot_model(self.model, to_file='model.png')\n",
    "        \n",
    "        print(\"Model built. Saved model.png\\n\")\n",
    "        \n",
    "    def train(self):\n",
    "        \"\"\"Default training\"\"\"\n",
    "\n",
    "        self.f1_test_history = []\n",
    "        self.f1_dev_history = []\n",
    "\n",
    "        for epoch in range(self.epochs):    \n",
    "            print(\"Epoch {}/{}\".format(epoch, self.epochs))\n",
    "            for i,batch in enumerate(iterate_minibatches(self.train_batch,self.train_batch_len)):\n",
    "                labels, tokens, casing,char = batch       \n",
    "                self.model.train_on_batch([tokens, casing,char], labels)\n",
    "\n",
    "            # compute F1 scores\n",
    "            predLabels, correctLabels = self.tag_dataset(self.test_batch, self.model)\n",
    "            pre_test, rec_test, f1_test = compute_f1(predLabels, correctLabels, self.idx2Label)\n",
    "            self.f1_test_history.append(f1_test)\n",
    "            print(\"f1 test \", round(f1_test, 4))\n",
    "\n",
    "            predLabels, correctLabels = self.tag_dataset(self.dev_batch, self.model)\n",
    "            pre_dev, rec_dev, f1_dev = compute_f1(predLabels, correctLabels, self.idx2Label)\n",
    "            self.f1_dev_history.append(f1_dev)\n",
    "            print(\"f1 dev \", round(f1_dev, 4), \"\\n\")\n",
    "            \n",
    "        print(\"Final F1 test score: \", f1_test)\n",
    "            \n",
    "        print(\"Training finished.\")\n",
    "            \n",
    "        # save model\n",
    "        self.modelName = \"{}_{}_{}_{}_{}_{}_{}\".format(self.epochs, \n",
    "                                                        self.dropout, \n",
    "                                                        self.dropout_recurrent, \n",
    "                                                        self.lstm_state_size,\n",
    "                                                        self.conv_size,\n",
    "                                                        self.learning_rate,\n",
    "                                                        self.optimizer.__class__.__name__\n",
    "                                                       )\n",
    "        \n",
    "        modelName = self.modelName + \".h5\"\n",
    "        self.model.save(modelName)\n",
    "        print(\"Model weights saved.\")\n",
    "        \n",
    "        self.model.set_weights(self.init_weights)  # clear model\n",
    "        print(\"Model weights cleared.\")\n",
    "\n",
    "    def writeToFile(self):\n",
    "        \"\"\"Write output to file\"\"\"\n",
    "\n",
    "        # .txt file format\n",
    "        # [epoch  ]\n",
    "        # [f1_test]\n",
    "        # [f1_dev ]\n",
    "        \n",
    "        output = np.matrix([[int(i) for i in range(self.epochs)], self.f1_test_history, self.f1_dev_history])\n",
    "\n",
    "        fileName = self.modelName + \".txt\"\n",
    "        with open(fileName,'wb') as f:\n",
    "            for line in output:\n",
    "                np.savetxt(f, line, fmt='%.5f')\n",
    "                \n",
    "        print(\"Model performance written to file.\")\n",
    "\n",
    "    print(\"Class initialised.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Set parameters\"\"\"\n",
    "\n",
    "EPOCHS = 30               # paper: 80\n",
    "DROPOUT = 0.5             # paper: 0.68\n",
    "DROPOUT_RECURRENT = 0.25  # not specified in paper, 0.25 recommended\n",
    "LSTM_STATE_SIZE = 200     # paper: 275\n",
    "CONV_SIZE = 3             # paper: 3\n",
    "LEARNING_RATE = 0.0105    # paper 0.0105\n",
    "OPTIMIZER = Nadam()       # paper uses SGD(lr=self.learning_rate), Nadam() recommended\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model built. Saved model.png\n",
      "\n",
      "Epoch 0/30\n",
      "f1 test  0.4286\n",
      "f1 dev  0.4566 \n",
      "\n",
      "Epoch 1/30\n",
      "f1 test  0.6359\n",
      "f1 dev  0.6603 \n",
      "\n",
      "Epoch 2/30\n",
      "f1 test  0.7004\n",
      "f1 dev  0.7048 \n",
      "\n",
      "Epoch 3/30\n",
      "f1 test  0.7077\n",
      "f1 dev  0.716 \n",
      "\n",
      "Epoch 4/30\n",
      "f1 test  0.766\n",
      "f1 dev  0.7669 \n",
      "\n",
      "Epoch 5/30\n",
      "f1 test  0.7819\n",
      "f1 dev  0.7859 \n",
      "\n",
      "Epoch 6/30\n",
      "f1 test  0.7962\n",
      "f1 dev  0.8017 \n",
      "\n",
      "Epoch 7/30\n",
      "f1 test  0.7896\n",
      "f1 dev  0.791 \n",
      "\n",
      "Epoch 8/30\n",
      "f1 test  0.8085\n",
      "f1 dev  0.8219 \n",
      "\n",
      "Epoch 9/30\n",
      "f1 test  0.8239\n",
      "f1 dev  0.8332 \n",
      "\n",
      "Epoch 10/30\n",
      "f1 test  0.8155\n",
      "f1 dev  0.8397 \n",
      "\n",
      "Epoch 11/30\n",
      "f1 test  0.8251\n",
      "f1 dev  0.8372 \n",
      "\n",
      "Epoch 12/30\n",
      "f1 test  0.8237\n",
      "f1 dev  0.8441 \n",
      "\n",
      "Epoch 13/30\n",
      "f1 test  0.8359\n",
      "f1 dev  0.8527 \n",
      "\n",
      "Epoch 14/30\n",
      "f1 test  0.8447\n",
      "f1 dev  0.864 \n",
      "\n",
      "Epoch 15/30\n",
      "f1 test  0.8517\n",
      "f1 dev  0.8699 \n",
      "\n",
      "Epoch 16/30\n",
      "f1 test  0.8415\n",
      "f1 dev  0.8633 \n",
      "\n",
      "Epoch 17/30\n",
      "f1 test  0.8602\n",
      "f1 dev  0.8828 \n",
      "\n",
      "Epoch 18/30\n",
      "f1 test  0.8553\n",
      "f1 dev  0.872 \n",
      "\n",
      "Epoch 19/30\n",
      "f1 test  0.8504\n",
      "f1 dev  0.8707 \n",
      "\n",
      "Epoch 20/30\n",
      "f1 test  0.852\n",
      "f1 dev  0.8755 \n",
      "\n",
      "Epoch 21/30\n",
      "f1 test  0.8586\n",
      "f1 dev  0.885 \n",
      "\n",
      "Epoch 22/30\n",
      "f1 test  0.8642\n",
      "f1 dev  0.8823 \n",
      "\n",
      "Epoch 23/30\n",
      "f1 test  0.869\n",
      "f1 dev  0.8932 \n",
      "\n",
      "Epoch 24/30\n",
      "f1 test  0.8647\n",
      "f1 dev  0.893 \n",
      "\n",
      "Epoch 25/30\n",
      "f1 test  0.8666\n",
      "f1 dev  0.8938 \n",
      "\n",
      "Epoch 26/30\n",
      "f1 test  0.8632\n",
      "f1 dev  0.8997 \n",
      "\n",
      "Epoch 27/30\n",
      "f1 test  0.8632\n",
      "f1 dev  0.8925 \n",
      "\n",
      "Epoch 28/30\n",
      "f1 test  0.8711\n",
      "f1 dev  0.8991 \n",
      "\n",
      "Epoch 29/30\n",
      "f1 test  0.8692\n",
      "f1 dev  0.9032 \n",
      "\n",
      "Final F1 test score:  0.8691753121153508\n",
      "Training finished.\n",
      "Model weights saved.\n",
      "Model weights cleared.\n",
      "Model performance written to file.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Construct and run model\"\"\"\n",
    "\n",
    "cnn_blstm = CNN_BLSTM(EPOCHS, DROPOUT, DROPOUT_RECURRENT, LSTM_STATE_SIZE, CONV_SIZE, LEARNING_RATE, OPTIMIZER)\n",
    "cnn_blstm.loadData()\n",
    "cnn_blstm.addCharInfo()\n",
    "cnn_blstm.embed()\n",
    "cnn_blstm.createBatches()\n",
    "cnn_blstm.buildModel()\n",
    "cnn_blstm.train()\n",
    "cnn_blstm.writeToFile()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot learning curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VPW9//HXJ/u+kJUkhITdsCgaBcVdQVQKaq1otS7lirau17a39nfbulRv1VqvdbnttWrttW64UbSouICiArLKvoQtJGFJQvZ9+fz+OIcQgUASMpkk83k+HvPIzJmTyecwOu853/NdRFUxxhhjAPy8XYAxxpiew0LBGGNMCwsFY4wxLSwUjDHGtLBQMMYY08JCwRhjTAsLBWOMMS0sFIwxxrSwUDDGGNMiwNsFdFR8fLxmZGR4uwxjjOlVli9fXqSqCcfar9eFQkZGBsuWLfN2GcYY06uIyM727GfNR8YYY1pYKBhjjGlhoWCMMaZFr7umcCQNDQ3k5eVRW1vr7VJ6jJCQENLS0ggMDPR2KcaYXqRPhEJeXh6RkZFkZGQgIt4ux+tUleLiYvLy8sjMzPR2OcaYXqRPNB/V1tYSFxdngeASEeLi4uzMyRjTYR4NBRGZLCKbRCRHRO49wvMDReRTEVktIgtEJO04/tbxFdvH2L+HMaYzPBYKIuIPPAtcDGQB14hI1iG7PQ78n6qOAR4Efu+peowxptdprIO962Hdu7DgEShY6fE/6clrCqcBOaq6DUBEXgemAetb7ZMF3OPenw/M9mA9HuXv78/o0aNbHs+ePZvIyEiuvPJKli5dyo033sgzzzxzxN998sknmTlzJmFhYR3+u7Nnz2bYsGFkZR2at8aYLtXcDM2NEBDU9a9dXwVFm6FwMxRudO9vhP3bQZvcnQTCEyBlbNf//VY8GQqpwK5Wj/OAcYfs8y1wBfAn4HIgUkTiVLXYg3V5RGhoKKtWrfrOtqqqKn73u9+xdu1a1q5d2+bvPvnkk1x33XWdDoUpU6ZYKBjT1RpqnW/muYsgdzHsWgwNNTDwDBgyEYZOhPhh0NGmWlXYvw22fw7bv4C85VCWe/B5vwDoNxgSs2Dk5RA/HBKGQ9wQCOr4Z0RHebv30c+BZ0TkRuALIB9oOnQnEZkJzARIT0/vzvqOS3h4OGeeeSY5OTlt7vPUU09RUFDAeeedR3x8PPPnz2fevHncd9991NXVMXjwYP72t78RERHBvffey5w5cwgICGDSpElcccUVzJkzh88//5yHHnqIt99+m8GDB3fjERrTh9SUwK5vYOfXTggUrICmeue5+OGQdRkERcDWT2Hefzq36HQYeqETEplnQ3DEkV+7LN8JgAO38jxne2QKDDwdTr7e+eBPGA79BoG/97qSezIU8oEBrR6nudtaqGoBzpkCIhIBfF9VSw99IVV9DngOIDs7W4/2Rx94bx3rC8qPr/JDZKVEcd/3Rh51n5qaGk466SQAMjMzeffdd9v12nfeeSdPPPEE8+fPJz4+nqKiIh566CE++eQTwsPDefTRR3niiSe47bbbePfdd9m4cSMiQmlpKTExMUydOpUpU6Zw5ZVXHvdxGtOjNTXA1vlQW+Z8Oxe/gz85wmMUGmuddvnGWmis/+7jpjrnfl0l7F4F+9yWbb8Ap4lm3C2QfjoMGA/hcd+tpTQXcj6BLZ/A6lmw7EXwD3L2H3KhExAlOw6eDRS7XwxD+znPZd4Dg851AqCHdQrxZCgsBYaKSCZOGFwN/LD1DiISD+xX1WbgV8CLHqzHo47UfNQZixcvZv369UyYMAGA+vp6Tj/9dKKjowkJCWHGjBlMmTKFKVOmHPffMqZXqCqG5X+DpS9ARUEXvahAQAgEBENgqNtUcwWkj4fUU47dTBOTDtk/dm6N9U4TU87HTkh8/JuD+wVFwMAJzn6ZZ0PiSPDr2SMBPBYKqtooIrcDHwH+wIuquk5EHgSWqeoc4Fzg9yKiOM1Htx3v3z3WN/qeTlWZOHEir7322mHPffPNN3z66ae89dZbPPPMM3z22WdeqNCYbrJ3HSz+M6x50/lmP+g8mPKE07auCtoMuD8PPG69rfUHf0CIewtyfvoFdN039IAgGHSOc5v0EJTuckIiNsM54/BiU1BnePSagqrOBeYesu23re6/BbzlyRp6g8jISCoqKoiPj2f8+PHcdttt5OTkMGTIEKqqqsjPzyclJYXq6mouueQSJkyYwKBBg77zu8b0Cc1NsPlDJwx2LISAUDjxGhh3KySO8HZ17RMzwLn1Ut6+0NznZWRkUF5eTn19PbNnz2bevHmH9RSaOXMmkydPJiUlhfnz5/PSSy9xzTXXUFdXB8BDDz1EZGQk06ZNo7a2FlXliSeeAODqq6/m5ptv5qmnnuKtt96yC83G+/asgZWvwPrZzjfyqBT3lure3PvRqRCRBH7+znWClf+AJf8LpTshKg0ufMC5ABvWz9tH5FNE9ajXbXuc7OxsPXSRnQ0bNnDCCSd4qaKey/5d+qDN8+BfP3O+iY67BYZfCv494Ltd9X6nmWflP2DPauei67CLIDAcyvOhvMD52XjI1CviD5HJUFMKDVWQfoZzXCOm9Izj6kNEZLmqZh9rP/tXN6Y3aG6CBb+HL/4ACSc47dazrne+UZ86A06+4fAeMt1R09bPnCDYNNfpvpk8Bi5+DEb/4PBv+KpOt8/WIVFe4Nz8g+CUGyHlpO49BnMYCwVjerqqInh7BmxbAGOvg0sedz5EN30A3/wvfPoAfP4ojL4STrsF+o/xbD1FObDqFfj2NajY7XSzzJ4BY6+F5NFt/56IExRh/Y6+n/EqCwVjerJdS+HNG5xgmPq008Z+wAlTnNve9U44fPuG86194AQ4bWbXNMFU7IHdq50moT2rnfsl252xAEMmwsWPwrCLPTP1g/EKCwVjeiJVWPo8fPgr58LsjHltN60kZcH3/gQX3OeEwtK/OkESlQajLoewOAiOhKBI52dwhNN/PjjKuR8c6XTTLNkBu791A2CNEwBV+w7+ndhM5xv+qTOc5qHI5G75p+jLahua2FNWS2xYEJEhAfj5eX8gm4WCMV3twNw2uYshfzlEpzl92Puf5PS0OZb6KnjvLufC7bDJcPlfIDT22L8X1g8m3Amn3+Z061zyF1j0rNtnvwP8ApzrFkMudJqiksdA8igIie7Y63Sxkqp6Nu+tIC4iiIy4cAL8e/YgsLaUVtfz2cZ9fLRuD19sLqKmwZnZx99PiA0LJCYsiH5hQcSGB9IvPIjYsCD6hQcRExZE9sBYMuLDPVqfhYIxx6up0fl2nbv44ORpB75hB0VCfQV8ivOhmnGWO83BOc48N4cOoCrcDLN+5MySef5v4Mx7Oj4C1s8fRlzq3FShodqZyqGuwqml5X4l1JU7j+urnB5NyWMg8QRnwJeXqCoFZbWsyy9jXUE56wrKWV9QRkHZwZ5LQf5+DEoIZ1hSJMOSIhiWFMnw5EgGxIYd89u2qlJR10hJVT0l1Q2U1TQQFx7EwLgwIkM8M9Bsd1kNH6/fy0fr9rB4236ampXkqBB+kJ3G6NRoymudevZX1zs/q+rZXlTFitxSSqrqaWx2eon+1+WjLRR6i+OZOru1BQsW8Pjjj/P+++97slxzPBrrIfdr2LnICYG8ZU53SoCYgTD4fGe6hPTTnVk0q4vcidDceXA2uu9tRNLBgBh0jnNW8c/bnQ/k696Bwecdf60iEBTu3CKTjv/1upiqsrO4mm/zSt0AKGN9QTkl1Q2AU/6g+HCyM/oxMiWKYcmR7K+sZ/O+CjbvqWD5zhLmfHtw6ouQQD+GJkYyNCmChMhgyqobKKmup6TK/VndQGn1wQ/ZQ8WFB5EeF0ZGXDjp/cIYGBfGwLhwBsaFERce1KHFq3L2VfDRur3MW7eHb/PKABicEM4tZw9i0shkxqRGt6u5qHWIRYd6fnS0hUIXOZ6ps00vUb0flr0A3zwPlXsAcZpVxl57MASiUg7/vYhEp2fQaHfSwpKdTkBsc29r3jy4b9pp8IOXnIFdfVB1fSOr88pYkVvCip0lrMwtpbjKmYk0yN+P4cmRXDQymZEpUWSlRHNC/0jCgo7+MVVZ18iWvRVs2VvJpr0VbN5bwVc5RZRUNRATFkhsWBAxYYEMSYxwmmbCD2xz7keFBFJYUcfO/dXsLK5iZ3E132zfz+xV+bQexhURHEBCZDAHPsYPjZXWY75qG5rZU+6c2Zw4IIb/mDycSVnJDElsYxbVoxARokKcOruDhYIHtWfqbIAPP/yQu+++m7CwMM4888yW7VVVVdxxxx2sXbuWhoYG7r//fqZNm8b48eN54YUXGDnSmefp3HPP5fHHHyc7+5jjUkxnFOXA4v+BVa9CYw0MvgCm/DdkTOhcO3vsQIi93ulJpOosprLtc2cxlVNvPmpPnsq6Rp5fuI1Afz9GpkQxMiWahEjvNfUcjaqSV1LDitwSlu8sYUVuCRt2V9DkfksflBDOeSMSOTk9lpMGxDA0KYLATlwniAgOYGx6LGPT23HdpQNqG5rIK6lpCYqdxVUUVdW3hELrs4bW3/dFwE+EsekxTMxKon90aJfW5Wl9LxQ+uNfpOdGVkkfDxY8cdZfOTp1dW1vLzTffzGeffcaQIUOYPn16y3MPP/ww559/Pi+++CKlpaWcdtppXHjhhUyfPp1Zs2bxwAMPsHv3bnbv3m2B0NVUYceXzoXazR844wLGTIfxP3V6+3QVEacNP/HYI8/XF5Rz+6sr2F5c9Z1vsElRwYxKiWZkajQjU6IYlRpNSnSIV9bpziup5uucYr7aWsSircXsq3CmagkL8uekATH85JzBnDwwhrEDYokN79ndWEMC/RmSGNGpb/e9Wd8LBS/p7NTZGzduJDMzk6FDhwJw3XXX8dxzzwEwb9485syZw+OPPw44AZKbm8tVV13FpEmTeOCBB5g1a5atpdCVGuud9XAXPeNcPA6Lg3PudbphRiR6pSRV5ZUluTz4/npiwwJ57ebxZKVEsb6gnLUtF2PLmL9pHweaymPCAt2giGJUSjSjUqMZ2O/YF2E7an9VPV9vLeKrnGK+3lrEzuJqAOIjgjljcBynZvbj5PQYhidF9treQr6m74XCMb7R9yaqyttvv83w4cMPey4uLo7Vq1fzxhtv8Je//MUL1fUxJTthzSx3zv7dzkpb33sKxlzlzLffSY1Nzfj7Sae/tZfXNvCrt9fwrzW7OWdYAk9cdSJxEU5z0fhBcYwfdHBqi5r6JjbscXrrrMsvY21BGS9+uZ2GJicpIoIDyEo5EBJRjE6NZlBCBP7tCIrGpmYqahspr21gW1EVX+c4QbB+d3nLa48f1I8bTs9gwpB4hiVFeOVMxRy/vhcKvcyIESPYsWMHW7duZfDgwd9ZR+Giiy7i6aef5umnn0ZEWLlyJWPHOot2T58+nccee4yysjLGjPHwtAZ9VXkBrJsN696BvKXOtsxznJHDgy/o1GIoTc3K2vwyFm4p5IvNRazILWF4ciTXjhvItJNSCA9u//9yq/NKuf3VleSX1nDvxSOYedago37TDw3y5+T0WE5u1bZe39jMln0VrMsvZ21BGWvzy3j1m53UNjhjF0IC/cjqH0VWShT+IpTXNlJe00B5bQPlNY3uzwaq6r+7Sm6Qvx+nDIzl55OGccaQeMakRtuZQB9hs6R2kYiICCorKw/b3nrq7JiYmCNOnd36QvNZZ53F1q1bef/996mpqeHuu+/m66+/prm5mczMzJauqnv37iU1NZXf/OY33HfffUesqSf8u/Q4VUXOlM5r34WdXwHqXDMa9X1nkfTYjA6/5O6yGhZuLuKLLYVOrxe3O+Wo1CiyB/Zj8bZiNu6pICI4gGknpXDtuIFkpUS1+Xqqyt++2sHvP9hAQkQwT/9wLKcM7LrpoxubmtleVMWa/DLWumGxcXc5fn5uL5fQgJbeLi33QwOJCgkgKjSQ5KgQxqbHEhrUjoF4psdo7yypFgp9mP27uGpKYMP7sPZtZ5yANjnNQ6O+D6OugPihHXq52oYmFm8rZuGWIr7YXMiWfc6XgYTIYM4aGs85wxKYMCSeeLeZR1VZuauUVxbn8v7qAuoamxmbHsO14wYyZUx/QgIPfriWVtfzi7dW8/H6vVx4QhKP/2AMMWE9+4Ks6R1s6mzju5qbYd86Z1rnrZ/Bjq+gucGZu+fMu50wSMzq0HKMjU3NLNpWzJxVBXy4dg8VdY0EBfgxLrMfP8hO4+xhCQxPijxiO7qItDTr/GbKCbyzIp9Xluzk529+y4PvrePKUwbww3HplNc2cMerK9lXUctvpmTx4wkZ1i5vup2Fguk5DkzJENSJYfwVe2HbfDcI5h+cZiIxC8bf6izKnjK2Q0Fw4Bv+nFUFvL96N0WVdUQEB3DRyGSmjOnP+EFxHW5CiQkL4sdnZnLThAyWbN/PK0tyeXnxDl78ajt+Aqmxobx16xmcOCCmQ69rTFfpM6GgqvatqpVe0Syo6kzDvP0L2L7Q+Vm1z1mtKzLZuUUkQWR/Z4qGyP7u42QIi3e6jB44G9jrjhgPi3emhxh8vrPQe1T/Dpe1eW8F/1yVz5xvC9i1v4agAD8uGJHI1BNTOG9E4neaezpLRFp6DxVVZvHW8jxKqur56XlDumUqA2Pa0idCISQkhOLiYuLi4iwYcAKhuLiYkJAQb5dyuLK8gwGwYyGU7XK2RyTDoHOdxdmrip1uoZV7YfcqZ8bPhuojv55foDPFxAX3wZALIGl0p3oNldc28NqSXN5dmc/GPRX4CUwYEs9dFwxj0sgkj04xEB8RzK3n2NrapmfoE6GQlpZGXl4ehYWF3i6lxwgJCSEtLc3bZTgziG7+AHI+cYJg/zZne2g/yDzLaePPONu52NtWoKs6s3pW7nXComKvc0YRN9SZaqIzzU2u0up6XvxqBy99tZ3y2kZOTo/hgakjuWR0/x47fYQxntQnQiEwMJDMzExvl2Faqy2HFf/nzOlftstZ0GXgBGdun8yzIHFk+7/Ri0BIlHPrYE+hthRV1vH8wu28vGgHVfVNXDQyiTvOH8qoVO+uGWCMt/WJUDA9SFmeEwTL/+7M1Z9+hrNk49CLjn9pyC6wt7yW//18G69+s5O6xmamjEnh9vOGMDw50tulGdMjeP//UtM3FKxy5gta967T3JM1Dc64HVJP8XZlgDNR218+38qspXk0qXLZSan89LzBDE7wrcnOjDkWCwXTec3NsGWeEwY7FjqrjI27FcbdAjHpANQ1NvHet7uJCw9iwpB4ggKObyqEdQVlvLksj/e+dQaBRQQHEBESQGRIABHBzs/I4MDvbNu8t4J3VuQjAleeMoCfnDOY9LiwrvgXMKbPsVAwnZO7BObc7iwbGZUKE38Hp9zwnfUFvs4p4tf/XMu2QmdVsqgQp4//JWP6M2Fw+wOipKqef67KZ9ayPNbvLifI34+JWUkkRYVQWddAZV0jFbXObXdZLRW1DVTWNrbM1xMc4Md14wcy8+xBpMT0rrntjeluFgqm40p2wGtXQ3AkXPE8jLwM/A922dxXXsvDczfwz1UFpPcL44UbshGB91fv5sN1e3hzeR7RoYFMykri0jH9mTAk/rDFVZqalYVbCnlzWR4fr99LfVMzo1KjeGDqSKadlNKuqR+ampXKukYC/KRDE9EZ48vs/xTTMXWV8NoPnfmDfvQuxB3sX9/UrLy8aAd/nLeZusZm7rpgKD85d3DLYK/zRyRR19jEl1uK+Nfq3Xy41gmImLADAZFCakwo767M4+3l+ewpryU2LJBrx6fzg1MGHHUSuSPx9xMbCGZMB1komPZrbobZP4HCDXDtW98JhFW7SvnPd9ewrqCcs4bG8+C0UWTGHz5+IDjAnwtOSOKCE5yAWLi5iH+t2c3cNXuYtSwPAD+Bc4YlcN/3sjj/hESCA2w2TmO6i4WCab8v/gAb5sCkh53RwziDvx77aBOvfZNLYmQwz/7wZC4ZndyukeXBAf5cmJXEhVlJ1DY0sXBLEfkl1Vw8uj9JUT1wNLYxPsBCwbTPhvdhwX/BidfA6bc5q8KtyOf3czdQWtPAjAmZ3D1xGBGdbLsPCfRnYlZSFxdtjOkoCwVzbHvXw7u3OGMOpjzJ/uoGfjZrFfM3FXLKwFh+N21Uh9v7jTE9k0dDQUQmA38C/IHnVfWRQ55PB/4OxLj73Kuqcz1Zk+mg6v1OT6OgCJj+Csvyq7njtZUUV9bz4LSRXDduYJcvBm+M8R6PhYKI+APPAhOBPGCpiMxR1fWtdvs1MEtV/ywiWcBcIMNTNfmsxjqoKoToDk6Q19QIb94IFXtovuFf/HVlNY99tJK02FDe+ekZNk+QMX2QJ88UTgNyVHUbgIi8DkwDWoeCAgfaHaKBAg/W45u2LYD374H9W2HAODhtJpwwFQLascTjvP+E7Z9Tdckz3PkZfLpxI5eMTuaR74/x6FTSxhjv8WQopAK7Wj3OA8Ydss/9wDwRuQMIBy480guJyExgJkB6enqXF9onVRXBvF/Dt685y1Cecy+sfgPenuGsXZB9E5xyo7NgzZGseBmW/IW9I2dwxWep7Kso5IGpI7n+9IG2ZoUxfdjxTURz/K4BXlLVNOAS4GUROawmVX1OVbNVNTshIaHbi+xVVGHlK/DMqbDmTSpPu5tfJj3Hmd+M486EF/ji1GepicuCBb+H/x4Fb81wpqxovVLbrm/Qf91DXr/xnL3yPPz84O2fnMENZ9iawcb0dZ48U8gHBrR6nOZua20GMBlAVReJSAgQD+zzYF19V9EWeO9u2PklTWnjeCXh3/n9IqGpuZizhsazaHsJcypigX/jjJgr+WnEAsZtnEvg2reg/4lO01L66TS/fi1FEs+lBT/mnKz+/OEHJ9rIYGN8hCdDYSkwVEQyccLgauCHh+yTC1wAvCQiJwAhgC2f1lGNdfDlf8PCP6KBoawYcz+3bxjF7px6Lh3Tn3snj2BAvzBUlZx9lXyZU8RXOcX8ZFs8TXWTudz/S27e+wkZ/7wNgBpCub7+Qe6ccho/nmBnB8b4EvHkAu8icgnwJE530xdV9WEReRBYpqpz3B5HfwUicC46/4eqzjvaa2ZnZ+uyZcs8VnOvs30hvP/vULyF4syp3F36Axbu9ufEATH8dsoJnDKwX5u/2tjUzOr8Mr7OKeLLLYUE7vqay2QBC0PO54Yf3cTY9NhuPBBjjCeJyHJVzT7mfp4MBU+wUHDVVcKH98LKl2mMSufP4T/lj9vT6R8dwi8nj2DqiSkdHj9QU9/EuoIyhiVHWu8iY/qY9oaCjWjujfauhzdvQIu2sLj/ddySO5GGkhDumTiYm88aRGhQ5yaQCw3yJzuj7TMLY0zfZ6HQ26x8Bf3Xz6jzD+Nuv9/y0Y7hXHlyGj+/aLhNImeMOW4WCr1FfTXM/TmseoUNwSdxQ9lMUgZk8N5lo2xksTGmy1go9AaFm9BZ10PhJv7c/H3+XHMlP/veCfzo9Az8bd4hY0wXslDo6b59g6b37qaiKZA76n9J8PAL+WjaKFtr2BjjERYKPVVDDQ3v/4LAb19mefMIHgi6h9uvOZvJo9q3gI0xxnSGhUJPVJRD5T+uJaJ0I882TmXPyffw6sWjbFSxMcbjLBR6mMpNnxPw+nTqm/35Vdh9XDH9Rk61bqLGmG5iodCTqFL27s9pbo5g3ml/4/6LzrBF640x3crbs6SaVipXv0dq7WYWp81gxqVnWSAYY7qdnSn0FKpUf/Jf7G9OYPQlM71djTHGR9mZQg/RtOlDEis28EHsdYxIjfN2OcYYH2VnCj2BKhUfPUxFcwKZF87wdjXGGB9mZwo9Qc4nxJSs4dXgK7lgZJq3qzHG+DA7U/A2Vao/fpj9Gk/8hJts2gpjjFfZmYK3bf2UsH0r+atexpXjBnm7GmOMj7MzBW9SpeGz31OocXDStTZi2RjjdXam4E3b5hNYsIz/aZzKj84c6u1qjDHGzhS8RpXm+Y9QSBz5md9nSGKktysyxhg7U/Ca7Z/jl7eEZxq+x/VnDvd2NcYYA1goeIcqLHiUYr84lsRcyjnDErxdkTHGABYK3rFjIeR+zVN1U/jhGUPxs26oxpgewq4peMOCRykLiOM9ncgX2QO8XY0xxrSwM4XutuNL2PklT9VeyrTsQUQEWy4bY3oO+0Tqbp8/SmVgHK/Unc+Hp2d4uxpjjPkOO1PoTjsXwfYv+N/GKUwYnkZGfLi3KzLGmO+wUOhOnz9CbXAcf605l5smZHq7GmOMOYyFQnfJXQLbFvAPv2kMSIxjwhBbM8EY0/NYKHSHij3wyX00hMTxx5IzuXFCBiLWDdUY0/PYhWZP2rMGFv0PrHkTmhuZlfjvBNZGcPnYVG9XZowxR2Sh0NWamyHnE1j0DGz/HALDIPsm9mTdxG+f28G/nZlOWJD9sxtjeqZjfjqJSBjwMyBdVW8WkaHAcFV93+PV9SYNNbD6DefMoGgTRPaHC++Hk2+AsH689MFGVJUfnT7Q25UaY0yb2vOV9W/AcuB093E+8CZwzFAQkcnAnwB/4HlVfeSQ5/8bOM99GAYkqmpM+0rvISr3wdLnnVt1MSSPgSv+imZNY+v+euYvK2TB5s0s2bafSVnJpMWGebtiY4xpU3tCYbCqTheRawBUtVracZVURPyBZ4GJQB6wVETmqOr6A/uo6r+32v8OYGxHD8Cr1v8T3v43aGqA4RdTm30rX9YPZ8GWQubP/Yr80hoAhiVF8OMzM7n5LFtZzRjTs7UnFOpFJBRQABEZDNS14/dOA3JUdZv7e68D04D1bex/DXBfO163Z6irhLn/QV3sMOYMfZg5eaEseWk/9U3LCQvyZ8KQeH563mDOHZ5Iakyot6s1xph2aU8o3Ad8CAwQkVeACcCN7fi9VGBXq8d5wLgj7SgiA4FM4LN2vG7P8NWfoHIP1xTfyoq8KoYkCtefPpDzRiSSnRFLcIC/tys0xpgOO2oouM1EG4ErgPGAAHepalEX13E18JaqNrVRx0xgJkB6enoX/+lOKMtDv36a+QFnUR+XzcJrT2FAP7tWYIzp/Y46eE1VFZirqsWq+i9Vfb8DgZAPtJ4XOs3ddiRXA68dpY7nVDVbVbMTEnrAgjSfPIBLFjsNAAASA0lEQVRqM7+pvJLrx2dYIBhj+oz2jGheISKnduK1lwJDRSRTRIJwPvjnHLqTiIwAYoFFnfgb3S9vOayZxfzYqygNSubSMf29XZExxnSZ9lxTGAdcKyI7gSqcJiRV1TFH+yVVbRSR24GPcLqkvqiq60TkQWCZqh4IiKuB192zkp5NFT76Fc3hifzH3guYOjaFcFsPwRjTh7TnE+2izr64qs4F5h6y7beHPL6/s6/f7da9A7uW8M2oByheFsT0U3vA9Q1jjOlCxwwFVd0pIicCZ7mbFqrqt54tqwdqqIWP74fk0Ty6eywjkoUT06K9XZUxxnSpY15TEJG7gFeARPf2D3egmW9Z/CyU5bIj+9eszK9k+qkDbKZTY0yf057moxnAOFWtAhCRR3EuCj/tycJ6lIq9sPAJGH4pLxUMICgg12Y6Ncb0Se3pfSRA6/EDTe423zH/IWiso+78+3lnRR6TRyYTExbk7aqMMabLtXdCvCUi8q77+DLgBc+V1MPsWQMrXobxP+XD3eGU1zZy9akDjv17xhjTC7XnQvMTIrIAONPddJOqrvRoVT2FKnz0/yA0Fs75Ba+/vIkB/UIZP8iW0jTG9E3tWU9hPLBOVVe4j6NEZJyqLvF4dd626QPY/gVc/Ad2VAWxaFsxP580DD8/32o9M8b4jvZcU/gzUNnqcaW7rW9rrId5v4b4YZB9E7OW7cJP4MpTrOnIGNN3teeagrQebayqzSLS94fxLn0e9m+FH75JI/68uTyP84Ynkhwd4u3KjDHGY9pzprBNRO4UkUD3dhewzdOFeVX1fvj8ERh8PgydyPxNhRRW1DHdLjAbY/q49oTCrcAZODOcHlgTYaYni/K6BY9AXQVMehhEeGNpLgmRwZw3ItHblRljjEe1p/fRPpxJ63xDcxOs+DuceA0kZbG3vJbPNu7jlnMGE+jfngw1xpjeqz3TXDzm9jgKFJFPRaRQRK7rjuK8omIPNNZC6ikAvLU8j2aFq7Kt6cgY0/e156vvJFUtB6YAO4AhwC88WZRXlbkriMak09yszFq2i/GD+pEZH+7duowxphu0JxQONDFdCrypqmUerMf7SnOdnzHpLN5ezM7iaq62KbKNMT6iPaHwvohsBE4BPhWRBKDWs2V5UelO52f0AN5YuouokAAmj0r2bk3GGNNNjhkKqnovTu+jbFVtAKqBaZ4uzGtKd0FYPKWNAXywdg+Xj00lJNDf21UZY0y3aNcgNFXd3+p+Fc6ynH1TaS7EpDN7ZT71jc22upoxxqdYH8tDleaiMem8vnQXY9KiyUqJ8nZFxhjTbSwUWmtuhrI8Cv0T2binwrqhGmN8TqdCQURGdHUhPULVPmiqY1FxOCGBfkw9KcXbFRljTLfq7JnCvC6toqcodcYofFQQzCWj+hMVEujlgowxpnu1eaFZRJ5q6ykgxjPleJnbHTWnLparU6O9XIwxxnS/o/U+ugn4GVB3hOeu8Uw5XuYOXMvXeFJjQ71cjDHGdL+jhcJSYK2qfn3oEyJyv8cq8qayXdQHxVBVG0pqjIWCMcb3HC0UrqSNkcuqmumZcrysNJey4P4ApNmZgjHGBx3tQnOEqlZ3WyU9QWkuhf5JRAQHEB1qF5mNMb7naKEw+8AdEXm7G2rxLlUo3UVeczypMaGIiLcrMsaYbne0UGj9qTjI04V4XVURNNawtaGfXWQ2xviso4WCtnG/b3J7Hm2oibGLzMYYn3W0C80nikg5zhlDqHsf97Gqat+aFKjMCYXNdf24zM4UjDE+qs1QUFXfmi+69RgFO1MwxvgomxDvgNJcGgKjqCDMrikYY3yWR0NBRCaLyCYRyRGRe9vY5yoRWS8i60TkVU/Wc1SluygPccco2JmCMcZHtWuRnc4QEX/gWWAikAcsFZE5qrq+1T5DgV8BE1S1REQSPVXPMZXmUuSfRFCAH/ERwV4rwxhjvMmTZwqnATmquk1V64HXOXwZz5uBZ1W1BEBV93mwnrapQmku+SSQGhOKn5+NUTDG+CZPhkIqsKvV4zx3W2vDgGEi8pWILBaRyR6sp201JdBQxbaGOLvIbIzxaR5rPurA3x8KnAukAV+IyGhVLW29k4jMBGYCpKd7YM1kd8rsjTUxpGZaKBhjfJcnzxTygdbrWaa521rLA+aoaoOqbgc244TEd6jqc6qararZCQkJXV9p64Fr1vPIGOPDPBkKS4GhIpIpIkHA1cCcQ/aZjXOWgIjE4zQnbfNgTUfmrri2y8YoGGN8nMdCQVUbgduBj4ANwCxVXSciD4rIVHe3j4BiEVkPzAd+oarFnqqpTaW5NAaEU064nSkYY3yaR68pqOpcYO4h237b6r4C97g37ynbRWVoClSKraNgjPFpNqIZnDEKAcn4+wnJUSHersYYY7zGQgGgNJfdxJMcFUKAv/2TGGN8l30C1pRCXTnbG+0iszHGWCi43VE31Vp3VGOMsVBwQ2FddbSdKRhjfJ6FQpkzRmFnU7ydKRhjfJ6FQmkuTQGhlBBpZwrGGJ9noVCaS1VICmBjFIwxxkKhNJfiwGQAUuxMwRjj4ywUSnPZIwnERwQTEuhby1IbY8yhfDsUasuhtpQddpHZGGMAXw8Ft+fR5tpYW5fZGGPw9VBwxyisqYqyMwVjjMFCAYAdNsWFMcYAFgo0+wdTRJSFgjHGYKFAdag7RqGfhYIxxvh8KOwPcsYo2JmCMcb4eiiU7WKPJBIVEkBkSKC3qzHGGK/z3VCor4LqYnKb4kiNDfN2NcYY0yP4biiUOmMUttT1s6YjY4xx+XAoON1R11bH2ER4xhjj8uFQ2AnAlrpYO1MwxhiX74ZC2S6a/YIoJNrOFIwxxuW7oVCaS01YCoqfTXFhjDEunw6F0qAkwMYoGGPMAT4cCrvY65dESKAf/cKDvF2NMcb0CL4ZCg01ULWPXc3ORHgi4u2KjDGmR/DNUHDHKOTUx9rANWOMacU3Q6HMGaOwrjrGricYY0wrvhkK7sC19TZwzRhjvsNnQ0H9AthHrIWCMca04qOhsIvasP4042fNR8YY04qPhkIupUH9AWzgmjHGtOLRUBCRySKySURyROTeIzx/o4gUisgq9/ZvnqynRWkuhf6JBPgJiZEh3fInjTGmNwjw1AuLiD/wLDARyAOWisgcVV1/yK5vqOrtnqrjMI11ULmHvOAE+seE4O9nYxSMMeYAT54pnAbkqOo2Va0HXgemefDvtU9ZHgA5DbaOgjHGHMqToZAK7Gr1OM/ddqjvi8hqEXlLRAYc6YVEZKaILBORZYWFhcdXlTtl9obqGFJjbOCaMca05u0Lze8BGao6BvgY+PuRdlLV51Q1W1WzExISju8vuqOZ11RF2UVmY4w5hCdDIR9o/c0/zd3WQlWLVbXOffg8cIoH63GU5qLiz27tZ2MUjDHmEJ4MhaXAUBHJFJEg4GpgTusdRKR/q4dTgQ0erMdRtou6sGSa8CfNrikYY8x3eKz3kao2isjtwEeAP/Ciqq4TkQeBZao6B7hTRKYCjcB+4EZP1dOiNJfyYBujYIwxR+KxUABQ1bnA3EO2/bbV/V8Bv/JkDYcpzaUo+CREoH+0hYIxxrTm7QvN3auxHip2k0cCiZHBBAX41uEbY8yx+NanYnk+aDPbbIyCMcYckW+Fgjtl9saaGFtcxxhjjsC3QqHMGaOwujLKuqMaY8wR+FYolOai4seuJms+MsaYI/G5UGgITaKBAOuOaowxR+BjobCLilBnjIINXDPGmMP5WCjkUuSfDNjANWOMORLfCYWmRijPp4B4YsMCCQvy6Lg9Y4zplXwnFCoKQJvY3hhnZwnGGNMG3wmFA2MUamOt55ExxrTB50JhdUUUaTZwzRhjjsh3QqFiDwDbGuxMwRhj2uI7oXDWPay7fj31BNo1BWOMaYPvhAKQW+Ucrp0pGGPMkflUKOSX1gDYvEfGGNMGnwqFvJIawoP8iQ4N9HYpxhjTI/lUKOSX1pAaG4qIeLsUY4zpkXwrFEpq7HqCMcYchW+FQmmNjVEwxpij8JlQqKxrpKymwbqjGmPMUfhMKOSXOD2PrPnIGGPa5jOhkFdSDdiU2cYYczQ+EwotYxTsTMEYY9rkM6GQHBXCpKwk4iOCvV2KMcb0WD6z0sykkclMGpns7TKMMaZH85kzBWOMMcdmoWCMMaaFhYIxxpgWFgrGGGNaWCgYY4xpYaFgjDGmhYWCMcaYFhYKxhhjWoiqeruGDhGRQmBnJ389HijqwnJ6gr52TH3teKDvHVNfOx7oe8d0pOMZqKoJx/rFXhcKx0NElqlqtrfr6Ep97Zj62vFA3zumvnY80PeO6XiOx5qPjDHGtLBQMMYY08LXQuE5bxfgAX3tmPra8UDfO6a+djzQ946p08fjU9cUjDHGHJ2vnSkYY4w5Cp8JBRGZLCKbRCRHRO71dj3HS0R2iMgaEVklIsu8XU9niMiLIrJPRNa22tZPRD4WkS3uz1hv1tgRbRzP/SKS775Pq0TkEm/W2FEiMkBE5ovIehFZJyJ3udt75ft0lOPpte+TiISIyDci8q17TA+42zNFZIn7mfeGiAS16/V8oflIRPyBzcBEIA9YClyjquu9WthxEJEdQLaq9tq+1SJyNlAJ/J+qjnK3PQbsV9VH3PCOVdVferPO9mrjeO4HKlX1cW/W1lki0h/or6orRCQSWA5cBtxIL3yfjnI8V9FL3ycRESBcVStFJBD4ErgLuAd4R1VfF5G/AN+q6p+P9Xq+cqZwGpCjqttUtR54HZjm5Zp8nqp+Aew/ZPM04O/u/b/j/A/bK7RxPL2aqu5W1RXu/QpgA5BKL32fjnI8vZY6Kt2Hge5NgfOBt9zt7X6PfCUUUoFdrR7n0cv/Q8B50+eJyHIRmentYrpQkqrudu/vAZK8WUwXuV1EVrvNS72imeVIRCQDGAssoQ+8T4ccD/Ti90lE/EVkFbAP+BjYCpSqaqO7S7s/83wlFPqiM1X1ZOBi4Da36aJPUadts7e3b/4ZGAycBOwG/ujdcjpHRCKAt4G7VbW89XO98X06wvH06vdJVZtU9SQgDadlZERnX8tXQiEfGNDqcZq7rddS1Xz35z7gXZz/EPqCvW6774H2331erue4qOpe93/YZuCv9ML3yW2nfht4RVXfcTf32vfpSMfTF94nAFUtBeYDpwMxIhLgPtXuzzxfCYWlwFD3anwQcDUwx8s1dZqIhLsXyRCRcGASsPbov9VrzAFucO/fAPzTi7UctwMfnK7L6WXvk3sR8wVgg6o+0eqpXvk+tXU8vfl9EpEEEYlx74fidKjZgBMOV7q7tfs98oneRwBuF7MnAX/gRVV92MsldZqIDMI5OwAIAF7tjccjIq8B5+LM6LgXuA+YDcwC0nFmw71KVXvFxds2judcnCYJBXYAt7Rqi+/xRORMYCGwBmh2N/8/nHb4Xvc+HeV4rqGXvk8iMgbnQrI/zhf9War6oPs58TrQD1gJXKeqdcd8PV8JBWOMMcfmK81Hxhhj2sFCwRhjTAsLBWOMMS0sFIwxxrSwUDDGGNPCQsEYl4g0tZolc1VXzqYrIhmtZ081pqcKOPYuxviMGneqAGN8lp0pGHMM7toVj7nrV3wjIkPc7Rki8pk7idqnIpLubk8SkXfd+e2/FZEz3JfyF5G/unPez3NHnyIid7rz+68Wkde9dJjGABYKxrQWekjz0fRWz5Wp6mjgGZyR8QBPA39X1THAK8BT7vangM9V9UTgZGCdu30o8KyqjgRKge+72+8Fxrqvc6unDs6Y9rARzca4RKRSVSOOsH0HcL6qbnMnU9ujqnEiUoSzYEuDu323qsaLSCGQ1npKAXea5o9Vdaj7+JdAoKo+JCIf4izOMxuY3WpufGO6nZ0pGNM+2sb9jmg970wTB6/pXQo8i3NWsbTVzJbGdDsLBWPaZ3qrn4vc+1/jzLgLcC3ORGsAnwI/gZbFT6LbelER8QMGqOp84JdANHDY2Yox3cW+kRhzUKi7etUBH6rqgW6psSKyGufb/jXutjuAv4nIL4BC4CZ3+13AcyIyA+eM4Cc4C7cciT/wDzc4BHjKnRPfGK+wawrGHIN7TSFbVYu8XYsxnmbNR8YYY1rYmYIxxpgWdqZgjDGmhYWCMcaYFhYKxhhjWlgoGGOMaWGhYIwxpoWFgjHGmBb/HxhPLIaDzf8hAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(cnn_blstm.f1_test_history, label = \"F1 test\")\n",
    "plt.plot(cnn_blstm.f1_dev_history, label = \"F1 dev\")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"F1 score\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Label distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "B-ORG: 3.1%\n",
      "I-ORG: 1.82%\n",
      "B-MISC: 1.69%\n",
      "I-MISC: 0.57%\n",
      "B-LOC: 3.51%\n",
      "I-LOC: 0.57%\n",
      "B-PER: 3.24%\n",
      "I-PER: 2.22%\n",
      "O: 83.28%\n"
     ]
    }
   ],
   "source": [
    "cnn_blstm = CNN_BLSTM(EPOCHS, DROPOUT, DROPOUT_RECURRENT, LSTM_STATE_SIZE, CONV_SIZE, LEARNING_RATE, OPTIMIZER)\n",
    "cnn_blstm.loadData()\n",
    "\n",
    "category_count = {\"B-ORG\\n\": 0, \"I-ORG\\n\":0, \"B-MISC\\n\": 0, \"I-MISC\\n\":0, \"B-LOC\\n\": 0, \"I-LOC\\n\": 0, \"B-PER\\n\": 0, \"I-PER\\n\": 0, \"O\\n\": 0}\n",
    "total_count = 0\n",
    "\n",
    "for sentence in cnn_blstm.trainSentences:\n",
    "    for word in sentence:\n",
    "        if word[1] in category_count.keys():\n",
    "            category_count[word[1]] += 1\n",
    "            total_count += 1\n",
    "\n",
    "for category, count in category_count.items():\n",
    "    print(\"{}: {}%\".format(category.replace(\"\\n\", \"\"), round((count/total_count)*100, 2)))            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
